
from crawler_lib import *

# test for Larva

lv = Larva("http://www.ppmsg.com/siwameitui/", retry=2, local=False)
lv.cur_dir = "http://www.ppmsg.com/siwameitui/"

links = lv.spider(res_type=lv.RES_LINK,ptn='[0-9]+\.html',match_mode=True)

for url in links:
  print url
print '$ done Larva test.'

# test for Spider
sp = Spider(links, "test_out")
sp.threading_spider(res_type = sp.RES_LINK)
while sp.query_status()!=2:
  time.sleep(3)

print '$ done Spider test.'
